package page;

import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

/**
 * PageRetriever class retrieves web pages and stores
 * them for later analysis by the PageParser class.
 * @author Vladimir Dudnic, Derek Fleener, Michael Groenier
 */
public class PageRetriever implements Runnable
{
        private boolean cont;
       
        /** 
         * Queue of web pages waiting to be parsed. 
         */
        LinkedBlockingQueue<String> waiting;
       
        LinkedBlockingQueue<Document> ready;
       
        /** 
         * Set of URLs already visited. 
         */
        Set<String> visited;
       
        /**
         * Constructor that initializes with a starting address.
         *
         * @param starting_address The first address.
         * @throws IOException if not acceptable address.
         * @throws InterruptedException
         */
        public PageRetriever(String starting_address) throws IOException, InterruptedException
        {
                cont = true;
                waiting = new LinkedBlockingQueue<String>();
                visited = new HashSet<String>();
                ready = new LinkedBlockingQueue<Document>();
                waiting.put(starting_address);
                Document starting_doc = getNextPage();
                ready.put(starting_doc);
        }
       
        /**
         * Adds a collection of unvisited web pages.
         *
         * @param addresses The collection of addresses.
         * @throws InterruptedException
         */
        public void addAllPages(Collection<? extends String> addresses) throws InterruptedException 
        {
                for (String address : addresses) 
                {
                        waiting.put(address);
                }
        }
       
        /**
         * @return the next available web page's content.
         * @throws InterruptedException
         * @throws NoSuchElementException if queue is empty
         */
        public Document getNextPage() throws InterruptedException 
        {
                Document doc = null;
                boolean validURL = false;
                String next = waiting.take();
                while(visited.contains(next))
                {
                        next = waiting.take();
                }
               
                while (!validURL)
                {
                        try 
                        {
                                 doc = Jsoup.connect(next).get();
                                 validURL = true;
                        }
                        catch (IOException e)
                        {
                                next = waiting.take();
                        } 
                        catch (IllegalArgumentException e)
                        {
                                next = waiting.take();
                        } 
                        catch (NullPointerException e) 
                        {
                                next = waiting.take();
                        }
                }
                visited.add(next);
                return doc;
        }
       
        public Document getNextDocument() throws InterruptedException
        {
                return ready.take();
        }
       
        public int checkIfReady() 
        {
                return ready.size();
        }

        private void retrieve() throws InterruptedException 
        {
                if (!waiting.isEmpty())
                {
                        Document retrieved = getNextPage();
                        ready.put(retrieved);
                }
        }
       
        public void forceQuit()
        {
                cont = false;
        }
       /**
        * The main method used to run this application.
        * (Implements the runnable interface)
        */
        @Override
        public void run()
        {
                while (cont)
                {
                        try 
                        {
                                retrieve();
                        } 
                        catch (InterruptedException e) 
                        {
                                cont = false;
                        }
                       
                }
               
        }
}

